# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from kuaidaili_redis.items import KuaidailiRedisItem
import time

# 基于最原始的 CrawlSpider 运行 Redis-scrapy
class FirstcrawlspiderSpider(CrawlSpider):
    name = 'firstcrawlspider'
    allowed_domains = ['www.kuaidaili.com']
    # 起始 url
    start_urls = ['https://www.kuaidaili.com/free/intr/']

    # 如果不加follow参数，follow是有默认值的
    # 如果有callback处理，follow就是False
    # 如果没有callback处理，follow就是True
    rules = (
        Rule(LinkExtractor(restrict_xpaths=r'//div[@id="listnav"]//a'), callback='parse_item', follow=True),
    )

    def parse_item(self, response):
        print('开始爬取...')
        ip_list = response.xpath('//tbody/tr/td[1]/text()').extract()
        port_list = response.xpath('//tbody/tr/td[2]/text()').extract()
        for i in range(len(ip_list)):
            item = KuaidailiRedisItem()
            item['ip'] = ip_list[i]
            item['port'] = port_list[i]
            yield item
